#!/usr/bin/env python

#
# download file
#

__version = "0.1"

def decode(page):
    """
    Accept-Encoding 'gzip' 
    source: http://love-python.blogspot.com/2008/07/accept-encoding-gzip-to-make-your.html
    
    corrections of source:
    - in case not gzipped page just return content
    
    """
    import StringIO
    import gzip
    
    encoding = page.info().get("Content-Encoding")    
    if encoding in ('gzip', 'x-gzip', 'deflate'):
        content = page.read()
        if encoding == 'deflate':
            data = StringIO.StringIO(zlib.decompress(content))
        else:
            data = gzip.GzipFile('', 'rb', 9, StringIO.StringIO(content))
        content = data.read()
    else:
        content = page.read()
    return content


def get_html_source(url, referer = '', data = 0, cj = 0, retry_counter = 0, retry_reason=''):
    """ 
    gets the content of a page
    source: http://love-python.blogspot.com/2009/03/updated-python-code-for-get-html-source.html

    corrections of source:
    - delete some print lines
    - added new parameter retry_reason
    
    """
    import urllib2
    
    if retry_counter > 3:
        print 'Error! %s: %s' % (url, retry_reason)
        return '', ''
    try:
        if cj:
            opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
        else:
            opener = urllib2.build_opener()

        opener.addheaders = [('Referer', referer),
                 ('Content-Type', 'application/x-www-form-urlencoded'),
                 ('User-Agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.14) Gecko/20080404 Firefox/2.0.0.14'),
                 ('Accept-Encoding', 'gzip,deflate')]

        if data:
            # HTTP POST
            usock = opener.open(url, data)
        else:
            # HTTP GET
            usock = opener.open(url)
        
        content = decode(usock) 
        usock.close()                
        return content, cj
    except urllib2.HTTPError, e:
        print 'The server couldn\'t fulfill the request. for url: ', url
        print 'Error code: ', e.code
        return get_html_source(url, referer, data, cj, retry_counter + 1)            
    except urllib2.URLError, e:
        return get_html_source(url, referer, data, cj, retry_counter + 1, e.reason)
        
if __name__ == "__main__":
    from optparse import OptionParser
    
    parser = OptionParser(version="%prog, v." + __version)
    parser.add_option("-u", "--url", dest="url", help="url for downloading")
    (options, args) = parser.parse_args()
    
    if (options.url == None):
        parser.error("Incorrect number of arguments, \n\nuse -h or --help for detail\n")

    print get_html_source(options.url)
